package data.crawling;


import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;

import data.aws.AmazonItemSearch;

import websphinx.Crawler;
import websphinx.Element;
import websphinx.Link;
import websphinx.Page;
import websphinx.DownloadParameters;
import websphinx.Tag;

public class PageCrawler extends Crawler{

//	private ArrayList<String> terms;
	private ArrayList<HashMap<String, String>> cameras;

	public final static String identifier = "->";
	public final static String pair_separator = "$";
	public final static String seller_separator = "*";
	String currentMan;
	
	public PageCrawler(){		
//		terms = new ArrayList<String>();
		cameras =new ArrayList<HashMap<String, String> >();
		this.currentMan = "";
	}
	
	public ArrayList<HashMap<String, String>> getCameras() {
		return cameras;
	}

	
	public void setCurrentMan(String cman){
		this.currentMan = cman;
	}
	
	@Override 
	/*
	 * true if the link should be followed, false otherwise.
	 */
	public boolean shouldVisit(Link l){
		if(Builder.manualTBox){
			return false;
		}		
		if(Builder.buildAssertions){
			//root excluded
			//for test purpose, only select a few cameras
			if(l.getDepth()<= Builder.maxCrawlDepth && l.getDepth() != 0){
				String[] ps = l.getURL().toString().split("/");
				//System.out.println(ps[ps.length-1]);
				if(ps[ps.length-1].startsWith(this.currentMan.replaceAll("_", "").toLowerCase()+"_")
						){
//					System.out.println(l);
					//only crawl the pages that indicates a camera by this manufacturer.
					return true;
				}else if(ps[ps.length-1].startsWith("oly_")){
					//olympus cams have different links, e.g., oly_
					return true;
				}
			}
		}
		else{
			return true;
		}
		return false;
	}
	
	
	@Override 
	/*
	 * Default: does nothing!
	 */
	public void visit(Page page){
//		System.out.println(page.getLinks().length);
		if(page.getDepth() <= 0){
			return;
		}
		pt(page.getTitle() + ", depth:"+page.getDepth());
		if(!page.hasContent()){
			pt("No page content! This page is skipped.");
		}else{
			String title = page.getTitle();
			String name = title.substring(0, title.indexOf(':'));
			HashMap<String, String> dc = new HashMap<String, String>();
			dc.put(Builder.CAMERANAME, name);
			//get cam type, e.g., slr? compact?
			String url = page.getBase().toString();
			String shortname = url.substring( url.lastIndexOf('/')+1);
			dc.put(Builder.CAMERASHORTNAME, shortname);
			if(url.contains("/slrs/")){
				dc.put(Builder.CAMTYPE, Builder.DLSR);
			}else if(url.contains("/compacts/")){
				dc.put(Builder.CAMTYPE, Builder.DC);
			}else{
				dc.put(Builder.CAMTYPE, Builder.Hybrid);
			}
			//vendor info
			dc.put(Builder.PRICEVENDORINFO, new AmazonItemSearch().searchItem(name));
			Element summary = null;
			Element spectable = null;
			for(Tag tag:page.getTags()){
				if(tag.isStartTag() && tag.getTagName().compareToIgnoreCase(Tag.BODY) == 0){
					Element body = tag.getElement();
					Element maintop = body.getChild().getSibling().getSibling(); 
					Element maincontent = maintop.getChild().getChild().getChild().getChild();	
					Element res = this.findSubElement(maincontent, "summary", false);
					if(res != null){
						summary = res;
//						System.out.println(summary.getStartTag());
					}
					res = this.findSubElement(maincontent, "tabContent", false);
					if(res != null){
						spectable  = res.getChild().getSibling();
//						System.out.println(spectable.getStartTag());
					}
					break;
				}
			}//end of for
			
			if(Builder.buildAssertions) {
				this.getReviews(summary, dc);
				this.processSpecTable(spectable, dc, name);
				this.cameras.add(dc);
				
				//write out DC in a file
				FileWriter fstream = null;
			    BufferedWriter out = null;
				try {
					fstream = new FileWriter("dpreviewRawData.txt", true);
//					System.out.println(dc.size());
			        out = new BufferedWriter(fstream);
			        out.append(Builder.CAMERANAME+identifier+dc.get(Builder.CAMERANAME)+pair_separator);
			        out.append(Builder.PRICEVENDORINFO+identifier+dc.get(Builder.PRICEVENDORINFO)+pair_separator);
			        dc.remove(Builder.CAMERANAME);
			        dc.remove(Builder.PRICEVENDORINFO);
			        for(String f:dc.keySet()){
			        	out.append(f+identifier+dc.get(f)+pair_separator); 	
			        }
					out.newLine();
				} catch (IOException e) {
					e.printStackTrace();
				}finally{
					try {
						out.close();
					} catch (IOException e) {
						e.printStackTrace();
					}
				}
			}
		}		
		page.discardContent();		
	}
	
	private void getReviews(Element summary, HashMap<String, String> dc){
		//get review table
		Element re = this.findSubElement(summary, "userReviews", true);
		Element are =  this.findSubElement(summary, "amazonCustomerReviews", true);
		if(re != null){
			String num = re.getChild().getChild().toText();
			num = num.replace('(', ' ');
			num = num.replace(')', ' ');
			num = num.trim();
			String score = re.getChild().getSibling().toText();
			dc.put(Builder.USERREVIEWNUM, num);
			dc.put(Builder.USERREVIEW, score);
		}
		if(are!=null){
			String num = re.getChild().getChild().toText();
			num = num.replace('(', ' ');
			num = num.replace(')', ' ');
			num = num.trim();
			String score = re.getChild().getSibling().toText();
			dc.put(Builder.AMAZONUSERREVIEWNUM, num);
			dc.put(Builder.AMAZONUSERREVIEW, score);
		}
		//get release date
		Element date = this.findSubElement(summary, "priceRange", true);
		String rd = date.toText();
		rd = rd.substring(rd.indexOf(':')+1).trim();
		dc.put(Builder.RELEASEDATE, this.stringToDate(rd));
//		System.out.println(rd);
	}
	
	private Element findSubElement(Element e, String classstr, boolean recur){
		Element find = e.getChild();
		Element ret = null;
		while(find!= null){
			if(find.getStartTag().toString().contains(classstr)){
				ret = find;
			}else if(recur){
				ret = this.findSubElement(find, classstr, recur);
			}
			if(ret != null){
				break;
			}
			find = find.getSibling();
		}
		return ret;
	}

	public enum Months{
		JAN, FEB, MAR, APR, MAY, JUN, JUL, AUG, SEP, OCT, NOV, DEC
	}
	//parameter in the form May 19, 2004, output: 20000622
	public String stringToDate(String src){
		String v[] = src.split(",");
		if(v.length!=2){
			System.err.println("Given date cannot be parsed: "+src);
			System.exit(-1);
		}
		String vv[] = v[0].split(" ");
		int month = Months.valueOf(vv[0].toUpperCase()).ordinal()+1;
		String m = String.format("%02d", month);
		String y = v[1];
		return y+m+vv[1];
	}
	
	private void processSpecTable(Element e, HashMap<String, String> dc, String name){
		if(e==null){
			return;
		}
//		//vendor info
//		try {
//			Thread.sleep(1000*Builder.queryDelay);
//			dc.put(Builder.PRICEVENDORINFO, this.getVendorPrice(name));
//		} catch (InterruptedException e1) {
//			e1.printStackTrace();
//		}
		Element firstrow = e.getChild();
		//process each row
		while(firstrow!=null &&
				firstrow.getStartTag().getTagName().compareToIgnoreCase(Tag.TR) == 0 ){	
			if( firstrow.getChild().getStartTag().getHTMLAttribute("class").equals("groupLabel")){
				firstrow = firstrow.getSibling();
				continue;
			}
//			boolean tref = false;
			//get f, k, and help
			Element feature = firstrow.getChild();
			Element value = feature.getSibling();
//			Element help = feature.getSibling();
//			System.out.println("f-v: "+feature.toText()+"->"+value.toText());
//			String tboxref = null;
//			if(help.getChild() !=null){
//				tboxref = help.getChild().getHTMLAttribute("href");
//				if(tboxref != null){
//					tboxref = tboxref.substring(tboxref.indexOf('=')+1);
//					tref = true;				
//					//System.out.println(tboxrefstr);
//				}
//			}
			String f = feature.toText().trim();
			String v = value.toText().trim();
			if(Builder.NULLALLOWED && v.length()<1){
				v = Builder.NULL;
			}
//			if(f.compareToIgnoreCase("Max resolution") == 0){
//				dc.put(Builder.MAXRESOLUTION, v);
//			}else if(f.compareToIgnoreCase("Low resolution") == 0){
//				dc.put(Builder.LOWRESOLUTION, v);
//			}else if(f.equalsIgnoreCase("pixel density")){
//				if(!v.matches("^\\d+.+")){
//					v = Builder.NULL;
//				}
//				dc.put(f, v);
//			}
//			else{
////				if(tref){
////					//f = f + identifier + tboxref;
////				}
//				dc.put(f, v);
//			}
			dc.put(f, v);
			firstrow = firstrow.getSibling();
		}
	}
	
	
//	private String getVendorPrice(String name){
//		name = name.trim().replaceAll(" ", "+");
//		//System.out.println(name);
//		String queryurl = "http://www.google.com/products?q="+name;
//		PriceCrawler pc = new PriceCrawler();
//		try {
//			pc.setRoot(new Link(queryurl));
//			pc.setMaxDepth(0);
//			pc.setDepthFirst(true);
//			pc.setDomain(Crawler.SUBTREE);
//			pc.setLinkType(Crawler.HYPERLINKS);
//			pc.clear();
//			pc.run();
//
//		} catch (MalformedURLException e) {
//			e.printStackTrace();
//		}
//		//build the vendor table
//		String ret = "";
//		int count = 0;
//		System.out.println("Vendor: "+pc.vendors.size());
//		for(String[] s:pc.vendors){
//			if(count >= Builder.maxVendeor){
//				break;
//			}
//			count ++;
//			System.out.println("Vendor: "+name+": "+s[0]+s[1]+s[2]);
//			for(int i=0; i<s.length; i++){
//				ret += s[i]+ "@";
//			}
//			ret +="\t";
//		}		
//		//System.out.println(name+": "+ret);
//		return ret;
//	}
	
//	//must return a concept for reivews
//	private String getUserReview(Element review){
//		String raw = review.toString();
//		Element temp = review.getChild().getChild().getChild().getChild().getChild();
//		String overallrate;
//		if(temp == null){
//			//no rating at all
//			overallrate = "(= (role review"+Builder.VALUEFEATURE+") "+Builder.NULL+")";
//		}else{
//			String star = temp.getHTMLAttribute("style");
//			//Overrall rating scale
//			star = star.substring(star.indexOf("width:")+6, star.indexOf("%"));
//			int rate = (java.lang.Integer.parseInt(star));
//			overallrate = "(= (role review"+Builder.VALUEFEATURE+") "+java.lang.Double.toString(rate*0.05)+")";
//			//if detailed rating if available at all
//			int bi = raw.indexOf("/reviews/read_opinions.asp?");
//			int ei = raw.lastIndexOf("Read owners opinions");
//			raw = raw.substring(bi, ei-2);
//			String reviewurl = "http://www.dpreview.com"+raw;
//			ReviewCrawler rc = new ReviewCrawler();
//			try {
//				rc.setRoot(new Link(reviewurl));
//				rc.setMaxDepth(0);
//				rc.setDepthFirst(true);
//				rc.setDomain(Crawler.SUBTREE);
//				rc.setLinkType(Crawler.HYPERLINKS);
//				rc.clear();
//				rc.run();
//			} catch (MalformedURLException e) {
//				e.printStackTrace();
//			}
//			//build detailed reviews
//			if(rc.detailedReviews.size()>0){
//				//details available
//				overallrate = "(and ";
//				for(String s:rc.detailedReviews.keySet()){
//					overallrate += "(exists (role "+ Builder.normalizeString(s.trim())+") (= (role review"+Builder.VALUEFEATURE+") "
//					+Builder.normalizeString((rc.detailedReviews.get(s)).trim())+")) ";
//				}
//				overallrate +=")";
//			}
//		}
//		//System.out.println(overallrate);
//		return overallrate;
//	}
	
//	private void processMainContentTerms(Element e){
//		if(e==null){
//			return;
//		}
//		//get the focus of the page
//		terms.add(e.getChild().getChild().toText());		
//	}
	
	
	public DownloadParameters setDownloadParameters(){
		DownloadParameters dp = new DownloadParameters(); 
		dp.changeMaxPageSize(1024);//1MB
		dp.changeDownloadTimeout(30);
		return dp;
	}
	
	public void pt(String s){
		System.out.println(s);
	}
	
	public boolean inArray(Object o, Object[] os){
		return java.util.Arrays.asList(os).contains(o);
	}
	

	

	
	
	
}
